{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "作业一、默认learning_rate=0.0001,模型收敛；设置learning_rate=0.01,收敛很慢甚至可能不收敛。\n",
    "        tinymind模型地址：https://www.tinymind.com/executions/1iockkta  (forfunforlifeinceptionv4yliuExec)\n",
    "\t经过5小时在GPU上的训练，结果如下：\n",
    "2018-01-12 15:09:15.149603: I tensorflow/core/kernels/logging_ops.cc:79] eval/Recall_5[0.828125]\n",
    "2018-01-12 15:09:15.149698: I tensorflow/core/kernels/logging_ops.cc:79] eval/Accuracy[0.622314453]\n",
    "\n",
    "作业二、\n",
    "实现的是 【L=101， growth=24】包含4个 dense-block的DenseNet-BC 网络，每个block层数相同 为（101-5）/4 = 24层（包括【1，1】卷积和【3，3】卷积各12层）\n",
    "5：输入进入第一个dense-block前的卷积、池化层，4个dense-block间的3个transition层，和第4个dense-block与分类器之间的全连接层\n",
    "数据集是quiz已打包的tfrecord文件。\n",
    "开始参考convert_quiz.py，得出图片大小为【320，320】，所以 经过 5次 stride=2 的卷积（same padding）或池化后，图片大小为【10，10】。\n",
    "但程序运行时报错提示此时图片大小为【6，6】，所以推测输入图片大小为【192，192】，但未找到根据。如果不是这样，则说明我的理解或计算有误。\n",
    "但是 模型不收敛！！！\n",
    "分析：应该尝试不同的初始化函数，但还没时间做。\n",
    "另外，从作业提供的代码看，dense-block中每个卷积操作之后都有dropout操作，这在论文中似乎并未提及。这么大量的dropout操作会不会丢失太多信息？\n",
    "从作业代码分析，is_training=false, keep=1, 所以并没有执行dropout操作。从语义上不能理解为什么 is_training=false..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def bn_act_conv_drp(current, num_outputs, kernel_size, scope='block'):\n",
    "    \"\"\"\n",
    "    Args:\n",
    "        num_outputs: 输出通道数\n",
    "    \"\"\"\n",
    "    current = slim.batch_norm(current, scope=scope + '_bn')\n",
    "    current = tf.nn.relu(current)\n",
    "    current = slim.conv2d(current, num_outputs, kernel_size, scope=scope + '_conv')\n",
    "    current = slim.dropout(current, scope=scope + '_dropout')\n",
    "    return current\n",
    "\n",
    "def block(net, layers, growth, scope='block'):\n",
    "    \"\"\"\n",
    "    Args:\n",
    "        layers: dense block包含的网络层数\n",
    "        growth: 增长率\n",
    "    \"\"\"\n",
    "    for idx in range(layers):\n",
    "        \"\"\"\n",
    "        BN-ReLU-Conv(1x1)-DroupOut-BN-ReLU-Conv(3x3)-DroupOut\n",
    "        \"\"\"\n",
    "        bottleneck = bn_act_conv_drp(net, 4 * growth, [1, 1],\n",
    "                                     scope=scope + '_conv1x1' + str(idx))\n",
    "        tmp = bn_act_conv_drp(bottleneck, growth, [3, 3],\n",
    "                              scope=scope + '_conv3x3' + str(idx))\n",
    "        net = tf.concat(axis=3, values=[net, tmp])\n",
    "    return net\n",
    "\n",
    "def transition(net, num_outputs, scope='transition'):\n",
    "    net = bn_act_conv_drp(net, num_outputs, [1, 1], scope=scope + '_conv1x1')\n",
    "    net = slim.avg_pool2d(net, [2, 2], stride=2, scope=scope + '_avgpool')\n",
    "    return net \n",
    "\n",
    "def densenet(images, num_classes=1001, is_training=False,\n",
    "             dropout_keep_prob=0.8,\n",
    "             scope='densenet'):\n",
    "    \"\"\"Creates a variant of the densenet model.\n",
    "\n",
    "      images: A batch of `Tensors` of size [batch_size, height, width, channels].\n",
    "      num_classes: the number of classes in the dataset.\n",
    "      is_training: specifies whether or not we're currently training the model.\n",
    "        This variable will determine the behaviour of the dropout layer.\n",
    "      dropout_keep_prob: the percentage of activation values that are retained.\n",
    "      prediction_fn: a function to get predictions out of logits.\n",
    "      scope: Optional variable_scope.\n",
    "\n",
    "    Returns:\n",
    "      logits: the pre-softmax activations, a tensor of size\n",
    "        [batch_size, `num_classes`]\n",
    "      end_points: a dictionary from components of the network to the corresponding\n",
    "        activation.\n",
    "    \"\"\"\n",
    "    growth = 32\n",
    "    compression_rate = 0.5\n",
    "\n",
    "    def reduce_dim(input_feature):\n",
    "        return int(int(input_feature.shape[-1]) * compression_rate)\n",
    "\n",
    "    end_points = {}\n",
    "\n",
    "    with tf.variable_scope(scope, 'DenseNet', [images, num_classes]):\n",
    "        with slim.arg_scope(bn_drp_scope(is_training=is_training,\n",
    "                                         keep_prob=dropout_keep_prob)) as ssc:\n",
    "\n",
    "            net = images\n",
    "            net = slim.conv2d(net, 2*growth, 7, stride=2, scope='conv1')\n",
    "            net = slim.max_pool2d(net, 3, stride=2, padding='SAME', scope='pool1')\n",
    "            \n",
    "            net = block(net, 6, growth, scope='block1')\n",
    "            net = transition(net, reduce_dim(net), scope='transition1')\n",
    "            net = slim.avg_pool2d(net, [2, 2], stride=2, scope='avgpool1')\n",
    "\n",
    "            net = block(net, 12, growth, scope='block2')\n",
    "            net = transition(net, reduce_dim(net), scope='transition2')\n",
    "            \n",
    "            net = block(net, 24, growth, scope='block3')\n",
    "            net = transition(net, reduce_dim(net), scope='transition3')\n",
    "\n",
    "            net = block(net, 16, growth, scope='block4')\n",
    "            net = slim.batch_norm(net, scope='last_batch_norm_relu')\n",
    "            net = tf.nn.relu(net)\n",
    "\n",
    "            # Global average pooling.\n",
    "            net = tf.reduce_mean(net, [1, 2], name='pool2', keep_dims=True)\n",
    "            \n",
    "            biases_initializer = tf.constant_initializer(0.1)\n",
    "            net = slim.conv2d(net, num_classes, [1, 1], biases_initializer=biases_initializer, scope='logits')\n",
    "            \n",
    "            logits = tf.squeeze(net, [1, 2], name='SpatialSqueeze')\n",
    "            \n",
    "            end_points['Logits'] = logits\n",
    "            end_points['predictions'] = slim.softmax(logits, scope='predictions')\n",
    "\n",
    "    return logits, end_points\n",
    "\n",
    "\"\"\"Contains a variant of the densenet model definition.\"\"\"\n",
    "\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "slim = tf.contrib.slim\n",
    "\n",
    "\n",
    "def trunc_normal(stddev): return tf.truncated_normal_initializer(stddev=stddev)\n",
    "\n",
    "\n",
    "def bn_act_conv_drp(current, num_outputs, kernel_size, scope='block'):\n",
    "    current = slim.batch_norm(current, scope=scope + '_bn')\n",
    "    current = tf.nn.relu(current)\n",
    "    current = slim.conv2d(current, num_outputs, kernel_size, scope=scope + '_conv')\n",
    "    current = slim.dropout(current, scope=scope + '_dropout')\n",
    "    return current\n",
    "\n",
    "\n",
    "def block(net, layers, growth, scope='block'):\n",
    "    for idx in range(layers):\n",
    "        bottleneck = bn_act_conv_drp(net, 4 * growth, [1, 1],\n",
    "                                     scope=scope + '_conv1x1' + str(idx))\n",
    "        tmp = bn_act_conv_drp(bottleneck, growth, [3, 3],\n",
    "                              scope=scope + '_conv3x3' + str(idx))\n",
    "        net = tf.concat(axis=3, values=[net, tmp])\n",
    "    return net\n",
    "\n",
    "\n",
    "def densenet(images, num_classes=1001, is_training=False,\n",
    "             dropout_keep_prob=0.8,\n",
    "             scope='densenet'):\n",
    "    \"\"\"Creates a variant of the densenet model.\n",
    "\n",
    "      images: A batch of `Tensors` of size [batch_size, height, width, channels].\n",
    "      num_classes: the number of classes in the dataset.\n",
    "      is_training: specifies whether or not we're currently training the model.\n",
    "        This variable will determine the behaviour of the dropout layer.\n",
    "      dropout_keep_prob: the percentage of activation values that are retained.\n",
    "      prediction_fn: a function to get predictions out of logits.\n",
    "      scope: Optional variable_scope.\n",
    "\n",
    "    Returns:\n",
    "      logits: the pre-softmax activations, a tensor of size\n",
    "        [batch_size, `num_classes`]\n",
    "      end_points: a dictionary from components of the network to the corresponding\n",
    "        activation.\n",
    "    \"\"\"\n",
    "    growth = 24\n",
    "    compression_rate = 0.5\n",
    "\n",
    "    def reduce_dim(input_feature):\n",
    "        return int(int(input_feature.shape[-1]) * compression_rate)\n",
    "\n",
    "    end_points = {}\n",
    "\n",
    "    with tf.variable_scope(scope, 'DenseNet', [images, num_classes]):\n",
    "        with slim.arg_scope(bn_drp_scope(is_training=is_training,\n",
    "                                         keep_prob=dropout_keep_prob)) as ssc:\n",
    "            pass\n",
    "            ##########################\n",
    "            # Put your code here.\n",
    "            current = end_points['pre_conv2'] = slim.conv2d(images, 2*growth, [7, 7], stride=2, padding='same', scope='pre_conv2')\n",
    "            current = end_points['pre_pool2'] = slim.max_pool2d(current, [3, 3], stride=2, scope='pre_pool2')\n",
    "\n",
    "            current = end_points['block1'] = block(current, 12, growth, scope='lblock1')\n",
    "            \n",
    "            current = end_points['transition1_conv2'] = bn_act_conv_drp(current, growth, [1, 1], scope='transition1_conv2')\n",
    "            current = end_points['transition1_pool2'] = slim.avg_pool2d(current, [2, 2], stride=2, scope='transition1_pool2') \n",
    "            \n",
    "            current = end_points['block2'] = block(current, 12, growth, scope='lblock2')\n",
    "            \n",
    "            current = end_points['transition2_conv2'] = bn_act_conv_drp(current, growth, [1, 1], scope='transition2_conv2')\n",
    "            current = end_points['transition2_pool2'] = slim.avg_pool2d(current, [2, 2], stride=2, scope='transition2_pool2') \n",
    "            \n",
    "            current = end_points['block3'] = block(current, 12, growth, scope='myblock3')\n",
    "\n",
    "            current = end_points['transition3_conv2'] = bn_act_conv_drp(current, growth, [1, 1], scope='transition3_conv2')\n",
    "            current = end_points['transition3_pool2'] = slim.avg_pool2d(current, [2, 2], stride=2, scope='transition3_pool2') \n",
    "\n",
    "            current = end_points['block4'] = block(current, 12, growth, scope='lblock3')\t\n",
    "            \n",
    "            current = end_points['global_pool2'] = slim.avg_pool2d(current, [6, 6], scope='lglobal_pool2') \n",
    "            current = end_points['PreLogitsFlatten'] = slim.flatten(current, scope='PreLogitsFlatten')\n",
    "            logits = end_points['Logits'] = slim.fully_connected(current, num_classes, activation_fn=None, scope='lLogits')\n",
    "\n",
    "            end_points['Predictions'] = tf.nn.softmax(logits, name='Predictions')\n",
    "            ##########################\n",
    "\n",
    "    return logits, end_points\n",
    "\n",
    "\n",
    "def bn_drp_scope(is_training=True, keep_prob=0.8):\n",
    "    keep_prob = keep_prob if is_training else 1\n",
    "    with slim.arg_scope(\n",
    "        [slim.batch_norm],\n",
    "            scale=True, is_training=is_training, updates_collections=None):\n",
    "        with slim.arg_scope(\n",
    "            [slim.dropout],\n",
    "                is_training=is_training, keep_prob=keep_prob) as bsc:\n",
    "            return bsc\n",
    "\n",
    "\n",
    "def densenet_arg_scope(weight_decay=0.004):\n",
    "    \"\"\"Defines the default densenet argument scope.\n",
    "\n",
    "    Args:\n",
    "      weight_decay: The weight decay to use for regularizing the model.\n",
    "\n",
    "    Returns:\n",
    "      An `arg_scope` to use for the inception v3 model.\n",
    "    \"\"\"\n",
    "    with slim.arg_scope(\n",
    "        [slim.conv2d],\n",
    "        weights_initializer=tf.contrib.layers.variance_scaling_initializer(\n",
    "            factor=2.0, mode='FAN_IN', uniform=False),\n",
    "        activation_fn=None, biases_initializer=None, padding='same',\n",
    "            stride=1) as sc:\n",
    "        return sc\n",
    "\n",
    "\n",
    "densenet.default_image_size = 224\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
